{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>adult_bmi</th>\n",
       "      <th>child_weight</th>\n",
       "      <th>child_height</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>75.00</td>\n",
       "      <td>M</td>\n",
       "      <td>27.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.67</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.5</td>\n",
       "      <td>70.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>70.00</td>\n",
       "      <td>M</td>\n",
       "      <td>21.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>3.00</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13.0</td>\n",
       "      <td>92.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>69.00</td>\n",
       "      <td>M</td>\n",
       "      <td>23.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   patient_id    age sex  adult_bmi  child_weight  child_height\n",
       "0           1  75.00   M       27.7           NaN           NaN\n",
       "1           2   0.67   M        NaN           9.5          70.0\n",
       "2           3  70.00   M       21.0           NaN           NaN\n",
       "3           4   3.00   M        NaN          13.0          92.0\n",
       "4           5  69.00   M       23.4           NaN           NaN"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_df = pd.read_csv('../data/origin/train_demographic_info.csv',\n",
    "                    names=['patient_id', 'age', 'sex', 'adult_bmi', 'child_weight', 'child_height'])\n",
    "x_train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>COPD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Bronchiolitis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Pneumonia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Bronchiolitis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>COPD</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   patient_id      diagnosis\n",
       "0           1           COPD\n",
       "1           2  Bronchiolitis\n",
       "2           3      Pneumonia\n",
       "3           4  Bronchiolitis\n",
       "4           5           COPD"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_df = pd.read_csv('../data/origin/train_patient_diagnosis.csv', names=['patient_id', 'diagnosis'])\n",
    "y_train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>adult_bmi</th>\n",
       "      <th>child_weight</th>\n",
       "      <th>child_height</th>\n",
       "      <th>diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>75.00</td>\n",
       "      <td>M</td>\n",
       "      <td>27.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COPD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.67</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.5</td>\n",
       "      <td>70.0</td>\n",
       "      <td>Bronchiolitis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>70.00</td>\n",
       "      <td>M</td>\n",
       "      <td>21.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pneumonia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>3.00</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>Bronchiolitis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>69.00</td>\n",
       "      <td>M</td>\n",
       "      <td>23.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COPD</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   patient_id    age sex  adult_bmi  child_weight  child_height      diagnosis\n",
       "0           1  75.00   M       27.7           NaN           NaN           COPD\n",
       "1           2   0.67   M        NaN           9.5          70.0  Bronchiolitis\n",
       "2           3  70.00   M       21.0           NaN           NaN      Pneumonia\n",
       "3           4   3.00   M        NaN          13.0          92.0  Bronchiolitis\n",
       "4           5  69.00   M       23.4           NaN           NaN           COPD"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.merge(x_train, y_train, how='inner', on='patient_id')\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train.fillna('', inplace=True)\n",
    "# train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_train_df = pd.DataFrame()\n",
    "old_path = '../data/origin/train'\n",
    "new_path = '../data/train'\n",
    "index = 0\n",
    "file_prefix_set = set()\n",
    "for file_name in os.listdir(old_path):\n",
    "    patient_id = file_name.split('_')[0]\n",
    "    file_prefix = file_name.split('.')[0]\n",
    "    if file_prefix in file_prefix_set:\n",
    "        continue\n",
    "    else:\n",
    "        file_prefix_set.add(file_prefix)\n",
    "    new_path_dir = new_path + '/' + str(index)\n",
    "    if not os.path.exists(new_path_dir):\n",
    "        os.makedirs(new_path_dir)\n",
    "#         print('{}/{}/{}'.format(old_path, dir_name, file_name))\n",
    "#         print('{}/{}'.format(new_path_dir, file_name))\n",
    "    os.system(\"cp {}/{}.txt {}/{}.txt\".format(old_path, file_prefix, new_path_dir, file_prefix))\n",
    "    os.system(\"cp {}/{}.wav {}/{}.wav\".format(old_path, file_prefix, new_path_dir, file_prefix))\n",
    "#         print(train[train['patient_id'] == int(patient_id)])\n",
    "    new_train_df = new_train_df.append(train_df[train_df['patient_id'] == int(patient_id)], ignore_index=True)\n",
    "#     patient_id_index = list(new_train.columns).index('patient_id')\n",
    "#     new_train.iloc[index, patient_id_index] = index\n",
    "    new_train_df.loc[index, 'patient_id'] = index\n",
    "#         print(new_train)\n",
    "    index += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>adult_bmi</th>\n",
       "      <th>child_weight</th>\n",
       "      <th>child_height</th>\n",
       "      <th>diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>764</th>\n",
       "      <td>764</td>\n",
       "      <td>58.0</td>\n",
       "      <td>F</td>\n",
       "      <td>24.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COPD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>765</th>\n",
       "      <td>765</td>\n",
       "      <td>68.0</td>\n",
       "      <td>M</td>\n",
       "      <td>27.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COPD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766</th>\n",
       "      <td>766</td>\n",
       "      <td>63.0</td>\n",
       "      <td>M</td>\n",
       "      <td>16.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COPD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>767</th>\n",
       "      <td>767</td>\n",
       "      <td>14.0</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Healthy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>768</th>\n",
       "      <td>768</td>\n",
       "      <td>75.0</td>\n",
       "      <td>M</td>\n",
       "      <td>27.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COPD</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     patient_id   age sex  adult_bmi  child_weight  child_height diagnosis\n",
       "764         764  58.0   F       24.7           NaN           NaN      COPD\n",
       "765         765  68.0   M       27.4           NaN           NaN      COPD\n",
       "766         766  63.0   M       16.5           NaN           NaN      COPD\n",
       "767         767  14.0   F        NaN           NaN           NaN   Healthy\n",
       "768         768  75.0   M       27.7           NaN           NaN      COPD"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_train_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_train_df['audio_and_txt_files_path'] = new_train['patient_id']\n",
    "new_train_df.to_csv('../data/train.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "COPD              655\n",
       "Pneumonia          37\n",
       "Healthy            29\n",
       "Bronchiectasis     16\n",
       "URTI               16\n",
       "Bronchiolitis      13\n",
       "LRTI                2\n",
       "Asthma              1\n",
       "Name: diagnosis, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_train_df['diagnosis'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>adult_bmi</th>\n",
       "      <th>child_weight</th>\n",
       "      <th>child_height</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>74.0</td>\n",
       "      <td>M</td>\n",
       "      <td>27.40</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>78.0</td>\n",
       "      <td>M</td>\n",
       "      <td>35.14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>69.0</td>\n",
       "      <td>M</td>\n",
       "      <td>28.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.5</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.26</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>71.0</td>\n",
       "      <td>M</td>\n",
       "      <td>34.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   patient_id   age sex  adult_bmi  child_weight  child_height\n",
       "0           1  74.0   M      27.40           NaN           NaN\n",
       "1           2  78.0   M      35.14           NaN           NaN\n",
       "2           3  69.0   M      28.00           NaN           NaN\n",
       "3           4   0.5   F        NaN          8.26          71.0\n",
       "4           5  71.0   M      34.00           NaN           NaN"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.read_csv('../data/origin/test_demographic_info.csv',\n",
    "                    names=['patient_id', 'age', 'sex', 'adult_bmi', 'child_weight', 'child_height'])\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test.fillna('', inplace=True)\n",
    "# test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 0\n",
    "new_test_df = pd.DataFrame()\n",
    "old_path = '../data/origin/test'\n",
    "new_path = '../data/test'\n",
    "file_prefix_set = set()\n",
    "ids = []\n",
    "for file_name in os.listdir(old_path):\n",
    "    patient_id = file_name.split('_')[0]\n",
    "    file_prefix = file_name.split('.')[0]\n",
    "    if file_prefix in file_prefix_set:\n",
    "        continue\n",
    "    else:\n",
    "        file_prefix_set.add(file_prefix)\n",
    "    new_path_dir = new_path + '/' + str(index)\n",
    "    if not os.path.exists(new_path_dir):\n",
    "        os.makedirs(new_path_dir)\n",
    "#     os.system(\"cp {}/{}.txt {}/{}.txt\".format(old_path, file_prefix, new_path_dir, file_prefix))\n",
    "#     os.system(\"cp {}/{}.wav {}/{}.wav\".format(old_path, file_prefix, new_path_dir, file_prefix))\n",
    "    new_test_df = new_test_df.append(test_df[test_df['patient_id'] == int(patient_id)], ignore_index=True)\n",
    "#     patient_id_index = list(new_test.columns).index('patient_id')\n",
    "#     new_test.iloc[index-origin_index, patient_id_index] = index\n",
    "    new_test_df.loc[index, 'patient_id'] = index\n",
    "    index += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_test_df['audio_and_txt_files_path'] = new_test['patient_id']\n",
    "new_test.to_csv('../data/input/test.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
